# -*- coding: utf-8 -*-
"""
__title__ = ''
__author__ = 'Yang'
__mtime__ = '2019/1/21 20:55'
# code is far away from bugs with the god animal protecting
    I love animals. They taste delicious.
              ┏┓      ┏┓
            ┏┛┻━━━┛┻┓
            ┃      ☃      ┃
            ┃  ┳┛  ┗┳  ┃
            ┃      ┻      ┃
            ┗━┓      ┏━┛
                ┃      ┗━━━┓
                ┃  神兽保佑    ┣┓
                ┃　永无BUG！   ┏┛
                ┗┓┓┏━┳┓┏┛
                  ┃┫┫  ┃┫┫
                  ┗┻┛  ┗┻┛
"""

import re
import time
import json
import requests
import random
import pymysql
import urllib.request

"""
create table zhanlang_comment(
url text,
name text,
place text,
yes_no text,
star text,
time text,
youyong text,
content text
)DEFAULT CHARSET=utf8;

"""

user_agent = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/64.0',
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/65.0',
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
              'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.6788.400 QQBrowser/10.3.2767.400',
              'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134']
cookie = [
    'll="118185"; bid=OiIqblmua28; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1548144985%2C%22https%3A%2F%2Faccounts.douban.com%2Faccounts%2Fpassport%2Fregister%22%5D; _pk_id.100001.8cb4=ba4c077cbddf0ad6.1548076295.2.1548144985.1548076295.; __utma=30149280.46616945.1548076295.1548076295.1548144986.2; __utmz=30149280.1548144986.2.2.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/accounts/passport/register; ps=y; ue="475525373@qq.com"; dbcl2="190417512:I6A3nOxY6IQ"; push_noty_num=0; push_doumail_num=0; _vwo_uuid_v2=D1A2AE29B37FF7ADEEB3ED30D28AB53A5|194be237ea160953addcbce6032df20d; ck=5LOq; ap_v=0,6.0; _pk_ses.100001.8cb4=*; __utmb=30149280.2.10.1548144986; __utmc=30149280; __utmt=1; __utmv=30149280.19041',
    '1l="118185"; bid=Hqkw7lzD3cc; ps=y; dbcl2="190384917:GgvCrKpPpHw"; _pk_id.100001.4cf6=aaffc69d178cb3e4.1548076686.2.1548146220.1548076693.; push_noty_num=0; push_doumail_num=0; __utma=30149280.2034681296.1548076689.1548076689.1548146202.2; __utmz=30149280.1548146202.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic|utmctr=%E8%B1%86%E7%93%A3https%3A%2F%2Fwww.baidu.com; __utmv=30149280.19038; __utma=223695111.404488756.1548076690.1548076690.1548146204.2; __utmz=223695111.1548146204.2.2.utmcsr=douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/note/497867310/; _vwo_uuid_v2=DC5A57F5315EE82992DE49F92CA55BFFA|b39f9613df4cb477e05fd495141946d8; douban-profile-remind=1; ck=760n; douban-fav-remind=1; ap_v=0,6.0; __utmb=30149280.4.10.1548146202; __utmc=30149280; __utmt=1; _pk_ref.100001.4cf6=%5B%22%22%2C%22%22%2C1548146204%2C%22https%3A%2F%2Fwww.douban.com%2Fnote%2F497867310%2F%22%5D; _pk_ses.100001.4cf6=*; __utmb=223695111.0.10.1548146204; __utmc=223695111',
    ''
]
header = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"
}
head = {
    'method': 'GET',
    'scheme': 'https',
    'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'accept-encoding': 'gzip, deflate, br',
    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,la;q=0.7',
    'upgrade-insecure-requests': '1',
}

url = 'https://movie.douban.com/subject/26363254/comments?start={0}&limit=20&sort=new_score&status={1}'
# url = 'https://movie.douban.com/subject/26363254/comments?start={0}&limit=20&sort=new_score&status={1}&comments_only=1'
proxy = [
    {'https': '117.114.149.66:53281'},
    {'https': '119.101.117.88:9999'},
    {'https': '117.70.38.142:9999'},
    {'https': '183.236.238.6:54152'},

]


# {'https': '117.70.38.142:9999'}

# {'https': '47.105.168.201:8118'}
# {'https': '59.57.151.126:31689'}
# {'https': '117.114.149.66:53281'}
# {'https': '222.92.112.69:8080'}
# {'https': '58.251.49.4:58729'}

def request_url(url):
    head['user-agent'] = random.choice(user_agent)
    ip = random.choice(proxy)
    # ip = proxy[0]
    while True:
        try:
            ip = random.choice(proxy)
            page = requests.get(url=url, headers=head, proxies=ip, timeout=5000)
            break
        except:
            print(ip)

    # page.encoding = 'utf-8'
    # page.decode('utf-8')
    return page.text


def isInstance(l):
    if isinstance(l, list):
        try:
            return l[0]
        except:
            return ''


def parse_html(page_html, db):
    with open('comment.csv', 'a', encoding='utf8') as f:
        s = re.findall('<div.class="comment">(.*?)</div>', page_html)
        # cousor = db.cursor()
        for i in s:
            start_time = time.time()
            div_ = str(i).replace('  ', '')
            flag = {}
            user = list(re.findall('<a.href="(https://www.douban.com/people/.*?)" class="">(.*?)</a>', div_)[0])
            user_infor_page_html = request_url(user[0])
            flag['bplace'] = isInstance(re.findall('<div.class="user-info">常居:&nbsp;<a.href=".*?">(.*?)</a>',
                                                   user_infor_page_html.replace('  ', '')))
            flag['yes_no'] = isInstance(re.findall('<span>(.*?)</span>', div_))
            flag['stat'] = isInstance(re.findall('<span.class="allstar(\d+).rating" title=".*?"></span>', div_))
            flag['time_'] = isInstance(re.findall('<span.class="comment-time " title="(.*?)">', div_))
            flag['votes'] = isInstance(re.findall('<span class="votes">(\d+)</span>', div_))
            flag['content'] = isInstance(re.findall('<span class="short">(.*?)</span>', div_))
            flag['user'] = user[1]
            flag['url'] = user[0]
            # print(user[1], bplace, yes_no, stat, time_, votes, content)
            print(flag['url'], flag['user'], flag['bplace'], flag['yes_no'], flag['stat'],
                  flag['time_'], flag['votes'], flag['content'])
            print(time.time() - start_time)
            # cousor.execute(
            #     "insert into zhanlang_comment values('{0}','{1}','{2}','{3}','{4}','{5}','{6}','{7}')".format(
            #         flag['url'],
            #         flag['user'],
            #         flag['bplace'],
            #         flag['yes_no'],
            #         flag['stat'],
            #         flag['time_'],
            #         flag['votes'],
            #         flag['content']))
            # db.commit()
            f.write(('{0},{1},{2},{3},{4},{5},{6},{7}\n').format(flag['url'],
                                                                 flag['user'],
                                                                 flag['bplace'],
                                                                 flag['yes_no'],
                                                                 flag['stat'],
                                                                 flag['time_'],
                                                                 flag['votes'],
                                                                 flag['content']))
            f.flush()
            # select * from zhanlang_comment


if __name__ == '__main__':
    # page_html = request_url(url=url.format(0, 'P'))
    # # page_html = request_url(url='https://www.ip.cn/')
    # print(page_html)
    for i in range(0, 481, 20):
        db = pymysql.connect(host='localhost', port=3306, user='root', password='123', db='pc')
        page_html = request_url(url=url.format(i, 'P'))
        parse_html(page_html, db=db)
        page_html = request_url(url=url.format(i, 'F'))
        parse_html(page_html, db=db)
        break
